import requests
from lxml import etree
from pymongo import MongoClient


# import asyncio
# from aiohttp import ClientSession
class HouseSpider:
    def __init__(self):
        self.list1 = list()
        self.headers = {
            "Host": "hangzhou.qfang.com",
            "Referer": "https://hangzhou.qfang.com/sale",
            "Sec-Fetch-Mode": "navigate",
            "Sec-Fetch-Site": "same-origin",
            "Sec-Fetch-Use": "?1",
            "Upgrade-Insecure-Requests": "1",
            "User-Agent": "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/76.0.3809.87 Safari/537.36 SLBrowser/6.0.1.8131"
        }

    def spider(self):

        for a in range(1, 11):
            url = "https://hangzhou.qfang.com/sale/f%s" % a

            #
            # async def get_house(url):
            #     async with ClientSession() as session:
            #         async with session.get(url=url, headers=headers) as resp:
            #             response = await resp.text()

            # with open("test1.txt", "w", encoding="utf-8") as f:
            #     f.write(r.content.decode())
            r = requests.get(url=url, headers=self.headers)
            html = etree.HTML(r.content.decode())
            title_list = html.xpath(
                '//div[@class="list-result"]/ul/li//div[@class="list-main-header clearfix"]/a/text()')
            # print(title_list)
            l1_list = html.xpath('//div[@class="list-result"]/ul/li//div[@class="house-metas clearfix"]/p[1]/text()')
            # print(l1_list)
            l2_list = html.xpath('//div[@class="list-result"]/ul/li//div[@class="house-metas clearfix"]/p[2]/text()')
            # print(l2_list)
            l3_list_1 = html.xpath('//div[@class="list-result"]/ul/li//div[@class="house-metas clearfix"]/p[4]/text()')
            l3_list_2 = []
            for i in l3_list_1:
                s = i.strip()
                l3_list_2.append(s)
            l4_list = html.xpath('//div[@class="list-result"]/ul/li//div[@class="house-metas clearfix"]/p[5]/text()')
            # print(l4_list)
            l5_list_1 = html.xpath('//div[@class="list-result"]/ul/li//div[@class="house-metas clearfix"]/p[6]/text()')
            l5_list_2 = []
            for i in l5_list_1:
                s = i.strip()
                l5_list_2.append(s)
            # print(l5_list_2)
            print(len(l5_list_2))
            l6_list_1 = html.xpath('//div[@class="list-result"]/ul/li//div[@class="text fl"]/a/text()')
            # print(l6_list_1)
            l7_list = html.xpath('//div[@class="list-result"]/ul/li//div[@class="list-price"]/p/span[1]/text()')
            # print(l7_list)
            l8_list = html.xpath('//div[@class="list-result"]/ul/li//div[@class="list-price"]/p[2]/text()')
            # print(l8_list)
            l9_list = html.xpath('//div[@class="list-result"]/ul/li//div[@class="house-tags clearfix"]/a/span/text()')
            # print(l9_list)
            print(len(l9_list))
            # l10_list = html.xpath('//div[@class="list-result"]/ul/li//div[@class="house-tags clearfix"]/p[1]/text()')
            # print(l10_list)
            # print(len(l10_list))

            for i in range(len(title_list)):
                dict1 = dict()
                dict1['房子标题'] = title_list[i]
                dict1['大小'] = l1_list[i]
                dict1['面积'] = l2_list[i]
                dict1['楼层'] = l3_list_2[i]
                dict1['朝向'] = l4_list[i]
                # dict1['建造日期'] = l5_list_2[i]
                dict1['位置'] = l6_list_1[i]
                dict1['总价'] = l7_list[i]
                dict1['价格'] = l8_list[i]
                # dict1['附近学校'] = l9_list[i]
                self.list1.append(dict1)
            print(len(self.list1))
            # print(list1)

    def save_mongo(self):
        client = MongoClient("127.0.0.1", 27017)
        collection = client["test5"]["house"]
        ret = collection.insert_many(self.list1)


if __name__ == '__main__':
    h = HouseSpider()
    h.spider()
    h.save_mongo()
